package edu.usc.cmb.iregulome.job;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Node;
import org.dom4j.io.DOMReader;

import edu.usc.cmb.iregulome.client.MainApp;
import edu.usc.cmb.iregulome.dao.StorageTableDAO;
import edu.usc.cmb.iregulome.entity.Sample;
import edu.usc.cmb.iregulome.entity.SampleAttribute;
import edu.usc.cmb.iregulome.entity.Study;
import edu.usc.cmb.iregulome.util.MLUtil;

/**
 * This runnable class is for download the annotation of experiment in a thread
 * 
 * @author hsy541
 * 
 */
public class AnnotationDownloadJob implements Runnable {

	private static BlockingQueue<String> annotationId = new LinkedBlockingQueue<String>();

	private static final String ftpHostname = "ftp.ncbi.nlm.nih.gov";

	private final HttpClient client = new HttpClient();

	private final DOMReader reader = new DOMReader();

	private static final Logger log = Logger
			.getLogger(AnnotationDownloadJob.class);

	private final static DocumentBuilderFactory dbf = DocumentBuilderFactory
			.newInstance();

	private static int otherErrorCount = 0;

	private static int parseErrorCount = 0;

	private static int withoutExperimentErrorCount = 0;

	private static final StorageTableDAO std = (StorageTableDAO) MainApp.ctx
			.getBean("storageTableDAO");

	static {
		dbf.setValidating(false);
		dbf.setNamespaceAware(false);
	}

	@Override
	public void run() {

		while (true) {
			String id = "";
			try {

				id = annotationId.take();

				System.out.println("download the sample " + id);
				// get the request string from which can extract the annotation
				String detailURLString = "http://www.ncbi.nlm.nih.gov/sites/entrez?db=sra&term="
						+ id + "&report=fullxml";

				String detailXml = "";
				// get detail xml
				{
					GetMethod getMethod = new GetMethod(detailURLString);
					client.executeMethod(getMethod);

					InputStream detailHtmlStream = getMethod
							.getResponseBodyAsStream();

					BufferedReader br = new BufferedReader(
							new InputStreamReader(detailHtmlStream));

					String lineHtml = br.readLine();
					while (lineHtml != null) {
						detailXml += lineHtml;
						lineHtml = br.readLine();
					}

					br.close();

					if (detailXml.indexOf("EXPERIMENT_PACKAGE") == -1) {
						log.error("Sample " + id
								+ "without experiment_package count: "
								+ (++withoutExperimentErrorCount));
						continue;
					}

					detailXml = detailXml.substring(detailXml
							.indexOf("EXPERIMENT_PACKAGE") - 4, detailXml
							.lastIndexOf("EXPERIMENT_PACKAGE")
							+ "EXPERIMENT_PACKAGE".length() + 4);
					detailXml = MLUtil.extractXMLFromHtml(detailXml);

					detailXml = detailXml.replaceAll("/<EXPERIMENT_PACKAGE>",
							"</EXPERIMENT_PACKAGE>");

				}

				// parse the xml

				Document doc = null;
				try {
					org.w3c.dom.Document root = dbf.newDocumentBuilder().parse(
							new ByteArrayInputStream(detailXml.getBytes()));

					doc = reader.read(root);
				} catch (Exception e) {
					log.error("Sample " + id + " parse error: ", e);
					log.error("parse error counter: " + (++parseErrorCount));
				}

				if (doc == null) {
					continue;
				}

				String strategy = gS(doc,
						"//EXPERIMENT_PACKAGE/EXPERIMENT/DESIGN/LIBRARY_DESCRIPTOR/LIBRARY_STRATEGY");

				String studyType = gS(doc,
						"//EXPERIMENT_PACKAGE/STUDY/DESCRIPTOR/STUDY_TYPE/@existing_study_type");
				// there is no LIBRARY_STRATEGY or LIBRARY_STRATEGY is not
				// ChIP-Seq
				// 0:skip;1:RNA-seq;2:CHIP-seq
				int flag = 0;
				if (strategy.equalsIgnoreCase("EST")
						|| studyType.equalsIgnoreCase("Transcriptome Analysis")) {
					// RNA-seq
					flag = 1;
				}
				if (strategy.equalsIgnoreCase("chip-seq")
						|| studyType.equalsIgnoreCase("Gene Regulation Study")
						|| studyType.equalsIgnoreCase("Epigenetics")) {
					// chio-seq
					flag = 2;
				}

				// get the sample value
				Sample sample = new Sample();

				String studyId = gS(doc,
						"//EXPERIMENT_PACKAGE/STUDY/@accession");
				String experimentId = gS(doc,
						"//EXPERIMENT_PACKAGE/EXPERIMENT/@accession");
				sample.setId(doc.selectSingleNode(
						"//EXPERIMENT_PACKAGE/SAMPLE/@accession")
						.getStringValue());
				sample.setAlias(gS(doc, "//EXPERIMENT_PACKAGE/SAMPLE/@alias"));
				sample
						.setTaxonid(Integer
								.valueOf(gS(
										doc,
										"//EXPERIMENT_PACKAGE/SAMPLE/SAMPLE_NAME/TAXON_ID",
										"0")));
				sample
						.setInstrumentModel(gS(doc,
								"//EXPERIMENT_PACKAGE/EXPERIMENT/PLATFORM/*/INSTRUMENT_MODEL"));
				sample.setDescription(gS(doc,
						"//EXPERIMENT_PACKAGE/SAMPLE/DESCRIPTION"));
				sample.setDownloadFlag(false);
				String url = ftpHostname + "/sra/Studies/"
						+ studyId.substring(0, 6) + "/" + studyId + "/"
						+ experimentId + "/";
				sample.setUrl(url);

				// get study value
				Study study = new Study();
				study.setId(studyId);
				study
						.setDiscription(gS(doc,
								"//EXPERIMENT_PACKAGE/STUDY/DESCRIPTOR/STUDY_DESCRIPTION"));

				if (flag == 1) {
					study.setType("RNA-seq");
				} else if (flag == 2) {
					study.setType("ChIP-seq");
				} else {
					study.setType("N/A");
				}
				study.setTitle(gS(doc,
						"//EXPERIMENT_PACKAGE/STUDY/DESCRIPTOR/STUDY_TITLE"));
				study
						.setStudyType(gS(doc,
								"//EXPERIMENT_PACKAGE/STUDY/DESCRIPTOR/STUDY_TYPE/@existing_study_type"));
				study.setLibraryStrategy(strategy);

				sample.setStudy(study);

				// prepare the attribute
				List<SampleAttribute> sampleAttrs = new LinkedList<SampleAttribute>();
				List<Node> attrs = doc
						.selectNodes("//EXPERIMENT_PACKAGE/SAMPLE/SAMPLE_ATTRIBUTES/SAMPLE_ATTRIBUTE");
				if (attrs != null) {
					for (Node node : attrs) {
						SampleAttribute sa = new SampleAttribute();
						sa.setTag(node.selectSingleNode("./TAG")
								.getStringValue());
						sa.setValue(node.selectSingleNode("./VALUE")
								.getStringValue());
						sa.setSample(sample);
						sampleAttrs.add(sa);
					}
				}
				List<Node> downloadNodes = doc
						.selectNodes("//EXPERIMENT_PACKAGE/RUN_SET/RUN");
				if (downloadNodes != null) {
					for (Node node : downloadNodes) {
						List<Node> downloadFiles = node
								.selectNodes("./DATA_BLOCK/FILES/FILE");
						if (downloadFiles == null) {
							continue;
						}
						for (Node node2 : downloadFiles) {
							SampleAttribute sa = new SampleAttribute();

							sa.setTag("DOWNLOAD_FILE");
							sa.setValue(node2.selectSingleNode("./@filename")
									.getStringValue());
							sa.setSample(sample);
							sampleAttrs.add(sa);
						}

					}
				}
				sample.setSampleAttributes(sampleAttrs);

				// save sample and study

				std.txSaveSample(sample);

			} catch (Exception e) {
				log.error("Sample " + id + " contain other error: ", e);
				log.error("Other error count: " + (++otherErrorCount));
			}
		}
	}

	private String gS(Document doc, String xPath, String nullString) {
		Node theNode = doc
				.selectSingleNode(xPath);
		if (theNode == null) {
			return nullString;
		} else {
			return theNode.getStringValue().trim();
		}
	}

	private String gS(Document doc, String xPath) {
		return gS(doc, xPath, "NULL");
	}

	public final static void addId(String id) {
		annotationId.offer(id);
	}
}
